const puppeteer = require('puppeteer');
const fs = require('fs');

let totalBlogs = [];
let errLogs = [];

async function main() {
  let tab = 'weekly';
  let sub_tab = '政采云';
  const fileName = `zhengcaiyun.json`;
  let totalBlogs = [];
  let blogs = [];

  const browser = await puppeteer.launch({headless: true, slowMo: 0});
  const page = (await browser.pages())[0];
  await page.setViewport({width: 1500, height: 0});

  page.on('pageerror', msg => {
    console.log(msg);
  });
  page.on('console', msg => {
    console.log(msg.text());
  });

  let url = `https://weekly.zoo.team/`;
  await page.goto(url, {waitUntil: 'domcontentloaded'});

  let arr = await page.$$eval('#root > div > main > div > ol > li', async (eles) => {
    let result = [];
    eles.forEach(async ele => {
      let url = ele.querySelector('a').href;
      result.push({url: url});
    });
    return result;
  });

  for (let i = 0; i < arr.length; i++) {
    let item = arr[i];
    // if (i > 0) {
    //   break;
    // }

    await page.goto(item['url'], {waitUntil: 'domcontentloaded'});
    await page.waitFor(1000);
    let create_at = await page.$eval('#root > div > main > div > div > h2 > time', el => el.innerText);
    create_at = create_at.replace(/.*?(\d+)\/(\d+)\/(\d+)/, '$3-$1-$2');

    blogs = await page.$$eval('#root > div > main > div > div > ol > li', async (eles, create_at) => {
      let tmpBlogs = [];
      eles.forEach(async ele => {
        let title = ele.querySelector('a > h4').innerHTML.replace(/(.+)<span class="item__tag">(.+)<\/span>/, '$1 ($2)');
        let content = ele.querySelector('a').href;
        let sub_tab = ele.querySelector('a > h4 > span').innerText;
        let intro = ele.querySelector('blockquote').innerText;
        tmpBlogs.push({
          title: title,
          content: content,
          tab: 'weekly',
          sub_tab: '政采云前端周刊',
          source: '政采云',
          intro: intro,
          create_at: create_at
        });
      });
      return tmpBlogs;
    }, create_at);

    console.log(i, arr.length, blogs.length);
    totalBlogs = totalBlogs.concat(blogs);
  }


  let writerStream = fs.createWriteStream(`crawler_data/${fileName}`);
  writerStream.write(JSON.stringify(totalBlogs));
  console.log('write done');

};


async function collectBlogs(page, tab, subTab) {
  let blogs = await page.$$eval('body > div.Central_wrap > div > div.Central_left.left > div > ul > li', async (eles, tab, sub_tab) => {
    let tmpBlogs = [];
    eles.forEach(async ele => {
      let title = ele.querySelector('div.rinfo > a').innerText;
      let content = ele.querySelector('div.rinfo > a').href;
      let intro = ele.querySelector('div.rinfo > p').innerText;
      let blog_date = ele.querySelector('div.rinfo > div > i').innerText;
      tmpBlogs.push({
        title: title,
        content: content,
        tab: tab,
        sub_tab: sub_tab,
        blog_date: blog_date
      });
    });
    return tmpBlogs;
  }, tab, subTab);
  return blogs;
}

main();


